#!/bin/bash

printf "\nWorking directory: $(pwd)\n"
printf "\nFormatting input files..\n"
echo "  $2"
gunzip -c $2 | perl -pe 's/,/:/g;' -e 's/:/,/' > individuals.tmp
# echo "  $3"
# perl -pe 's/,/:/g;' -e 's/:/,/;' -e 's/:/,/' $3 > households.tmp

echo "eID,aID" > households.header
# head -n1 $3 > households.header
gunzip -c $2 | head -n1 | sed -e 's/pid,//g' -e 's/,/_e,/g' -e 's/$/_e/' > individuals.header1
gunzip -c $2 | head -n1 | sed -e 's/pid,//g' -e 's/,/_a,/g' -e 's/$/_a/' > individuals.header2

paste -d, households.header individuals.header1 individuals.header2 > $1

rm households.header; rm individuals.header1; rm individuals.header2


printf "\nJoining..\n"
printf "  ego\n"
awk -F, 'FNR==NR{a[ $1 ] = $2; next}($1 in a){print $1, $2, a[ $1 ]}' OFS=, individuals.tmp $3 > merged1.tmp # households.tmp

# rm households.tmp

printf "  alter\n"
awk -F, 'FNR==NR{a[ $1 ] = $2; next}($2 in a){print $1, $2, $3, a[ $2 ]}' OFS=, individuals.tmp merged1.tmp > merged2.tmp

rm individuals.tmp
rm merged1.tmp

printf "\nFormatting output file..\n"
perl -pe 's/:/,/g;' merged2.tmp >> $1
rm merged2.tmp
gzip -c $1 > $1.gz && rm $1

printf "\nDone (EOF).\n\n"


# echo "eID,aID,eDOB_Cali07,aDOB_Cali07,OCC_Cali07,DOBdiff_Cali07,eGENsame_Cali07,aGENsame_Cali07,eDOB_Cali09,aDOB_Cali09,OCC_Cali09,DOBdiff_Cali09,eGENsame_Cali09,aGENsame_Cali09,eDOB_Cali10,aDOB_Cali10,OCC_Cali10,DOBdiff_Cali10,eGENsame_Cali10,aGENsame_Cali10,eDOB_Cali12,aDOB_Cali12,OCC_Cali12,DOBdiff_Cali12,eGENsame_Cali12,aGENsame_Cali12,eDOB_Cali14,aDOB_Cali14,OCC_Cali14,DOBdiff_Cali14,eGENsame_Cali14,aGENsame_Cali14,localitycode_cali307,registrantid_cali307,lastname_cali307,firstname_cali307,addrnum_cali307,addrnumsuff_cali307,stdirprefix_cali307,streetname_cali307,stdirsuffix_cali307,unitnumber_cali307,city_cali307,state_cali307,zip_cali307,language_cali307,dateofbirth_cali307,gender_cali307,party_cali307,status_cali307,statusreason_cali307,registrationdate_cali307,precinct_cali307,precinctpart_cali307,registrationmethodcode_cali307,assistanceflag_cali307,placeofbirth_cali307,sectiontownshiprangedirection_cali307,ss05_cali307,gp06_cali307,gg06_cali307,localitycode_cali309,registrantid_cali309,lastname_cali309,firstname_cali309,addrnum_cali309,addrnumsuff_cali309,stdirprefix_cali309,streetname_cali309,stdirsuffix_cali309,unitnumber_cali309,city_cali309,state_cali309,zip_cali309,language_cali309,dateofbirth_cali309,gender_cali309,party_cali309,status_cali309,statusreason_cali309,registrationdate_cali309,precinct_cali309,precinctpart_cali309,registrationmethodcode_cali309,assistanceflag_cali309,placeofbirth_cali309,sectiontownshiprangedirection_cali309,elec1_cali309,elec2_cali309,elec3_cali309,elec5_cali309,elec6_cali309,elec7_cali309,localitycode_cali310,registrantid_cali310,lastname_cali310,firstname_cali310,addrnum_cali310,addrnumsuff_cali310,stdirprefix_cali310,streetname_cali310,stdirsuffix_cali310,unitnumber_cali310,city_cali310,state_cali310,zip_cali310,language_cali310,dateofbirth_cali310,gender_cali310,party_cali310,status_cali310,statusreason_cali310,registrationdate_cali310,precinct_cali310,precinctpart_cali310,registrationmethodcode_cali310,assistanceflag_cali310,placeofbirth_cali310,sectiontownshiprangedirection_cali310,dist1_cali310,dist2_cali310,dist3_cali310,dist4_cali310,dist5_cali310,dist6_cali310,dist7_cali310,dist8_cali310,elec1_cali310,elec2_cali310,elec3_cali310,elec5_cali310,elec6_cali310,elec7_cali310,localitycode_cali312,registrantid_cali312,lastname_cali312,firstname_cali312,addrnum_cali312,addrnumsuff_cali312,stdirprefix_cali312,streetname_cali312,stdirsuffix_cali312,unitnumber_cali312,city_cali312,state_cali312,zip_cali312,language_cali312,dateofbirth_cali312,gender_cali312,party_cali312,status_cali312,statusreason_cali312,registrationdate_cali312,precinct_cali312,precinctpart_cali312,registrationmethodcode_cali312,assistanceflag_cali312,placeofbirth_cali312,sectiontownshiprangedirection_cali312,dist1_cali312,dist2_cali312,dist3_cali312,dist4_cali312,dist5_cali312,dist6_cali312,dist7_cali312,dist8_cali312,elec1_cali312,elec2_cali312,elec3_cali312,elec5_cali312,elec6_cali312,elec7_cali312,localitycode_cali314,registrantid_cali314,lastname_cali314,firstname_cali314,addrnum_cali314,addrnumsuff_cali314,stdirprefix_cali314,streetname_cali314,stdirsuffix_cali314,unitnumber_cali314,city_cali314,state_cali314,zip_cali314,language_cali314,dateofbirth_cali314,gender_cali314,party_cali314,status_cali314,statusreason_cali314,registrationdate_cali314,precinct_cali314,precinctpart_cali314,registrationmethodcode_cali314,assistanceflag_cali314,placeofbirth_cali314,sectiontownshiprangedirection_cali314,dist1_cali314,dist2_cali314,dist3_cali314,dist4_cali314,dist5_cali314,dist6_cali314,dist7_cali314,dist8_cali314,elec1_cali314,elec2_cali314,elec3_cali314,elec5_cali314,elec6_cali314,elec7_cali314" | gzip -c > formattedByDyad_CaliAllYears.csv.gz && awk -F, 'NR!=1' OFS=, formattedByDyad_CaliAllYears.csv | gzip -c >> formattedByDyad_CaliAllYears.csv.gz
# paste -d, formattedByIndividual_Cali307.csv.tmp.header formattedByIndividual_Cali309.csv.tmp.header formattedByIndividual_Cali310.csv.tmp.header formattedByIndividual_Cali312.csv.tmp.header formattedByIndividual_Cali314.csv.tmp.header | perl -pe 's/pid,//g;' -e 's/^/pid,/'